#include"mycrawl.h"

/*
1.如果C++调用一个C语言编写的.DLL时，当包括.DLL的头文件或声明接口函数时，应加extern "C" {　}。
2.在C中引用C++语言中的函数和变量时，C++的头文件需添加extern "C"，但是在C语言中不能直接引用声明了extern "C"的该头文件，
应该仅将C文件中将C++中定义的extern "C"函数声明为extern类型。
*/

struct html_parse *html_newparser(void);
int parser_html_get_hrefs(const char*, vector<string>&);

int get_page_through_url(char *url, http_resp *resp_p)
{
	int sock = 0;
	struct sockaddr_in sin;
    http_uri *uri_obj_p = NULL;
	struct hostent* hostent_p = NULL;
	string request_packet;
	int n;/*read返回值*/
	resp_packet_buf resp_rev_buf;
	char *realloc_buf_src = NULL;
	//http_resp *resp_p;
	int ret;
	int parse_resp_packet_ret;
	int received_bytes;

    memset(&sin, 0, sizeof(sin));
    sin.sin_family = AF_INET;
    sin.sin_port   = htons((unsigned short)80);

    uri_obj_p = http_uri_new();
    if (NULL == uri_obj_p){
    	fprintf(stderr,"http_uri_new error\n");
    	ret = -1;
    	goto end1;
    }

    if (-1 == http_uri_parse(url, uri_obj_p) ){
    	fprintf(stderr, "parse the URL error\n");
    	ret = -2;
    	goto end2;
	}

	hostent_p = gethostnameinfo(uri_obj_p, uri_obj_p->host);
	if ((NULL == hostent_p) || (NULL == hostent_p->h_addr)){
		ret = -3;
		goto end2;
	}
	/*valgrind说明这里有错误的拷贝*/
	memcpy(&sin.sin_addr, hostent_p->h_addr, hostent_p->h_length);

	sock = socket(PF_INET, SOCK_STREAM, 0);
	if (sock < 0){
		fprintf(stderr, "create sock error\n");
		ret = -4;
		goto end2;
	}

	if (connect(sock, (struct sockaddr *)&sin, sizeof(sin)) < 0){
		perror("CONNECT:");
		//fprintf(stderr, "connect sock error\n");
		ret = -5;
		goto end3;
	}

	request_packet =  "GET ";
	request_packet += uri_obj_p->resource;
	request_packet += " HTTP/1.1\r\n";
	request_packet += "Host: ";
	request_packet += uri_obj_p->host;
	request_packet += "\r\n";
	request_packet += "Connection: close\r\n\r\n";

	if ( request_packet.length() != write(sock, request_packet.c_str(), request_packet.length())){
		fprintf(stderr, "write the request to the sock error\n");
		ret = -6;
		goto end3;
	}

	resp_rev_buf.buf_size = DEFAULT_HTTP_RESP_REV_BUF_SIZE;
	resp_rev_buf.buf_src  = NULL;
	resp_rev_buf.buf_left = DEFAULT_HTTP_RESP_REV_BUF_SIZE;

	resp_rev_buf.buf_src  = (char *)malloc(DEFAULT_HTTP_RESP_REV_BUF_SIZE);
	if (NULL == resp_rev_buf.buf_src){
		fprintf(stderr, "malloc the resp. receive buffer error\n");
		ret = -7;
		goto end3;
	}
	/*分配的这片内存没有进行初始化，需要么？？？？？？*/

	n = 0;
	received_bytes = 0;
	/*如果缓冲区大小不够，需要扩大大小，该怎么办？*/
	while(1){

		errno = 0;

		//n = read(sock, resp_rev_buf.buf_src+resp_rev_buf.buf_size-resp_rev_buf.buf_left, resp_rev_buf.buf_left);
		//n = read(sock, resp_rev_buf.buf_src+resp_rev_buf.buf_size-resp_rev_buf.buf_left, resp_rev_buf.buf_left);
		n = read(sock, resp_rev_buf.buf_src + received_bytes, resp_rev_buf.buf_left);
		/*如果n等于resp_rev_buf.buf_left，那么说明缓冲区被全部读满了，那么我们需要多开辟空间
		  如果n小于resp_rev_buf.buf_left，那么说明，我们的缓冲区还有多，也可能是收到一个信号，这里也要进行处理。
		  如果n等于0，那么我们读到尾部了
		*/
		/*调整free缓冲区大小*/
		if (n == resp_rev_buf.buf_left){
			realloc_buf_src = (char *)realloc(resp_rev_buf.buf_src, resp_rev_buf.buf_size*2);

			if (NULL == realloc_buf_src){
				fprintf(stderr, "realloc the resp. receive buffer error\n");
				ret = -8;
				goto end4;
			}
			resp_rev_buf.buf_src = realloc_buf_src;

			/*剩下的大小和原来一样大*/
			resp_rev_buf.buf_left = resp_rev_buf.buf_size;
			resp_rev_buf.buf_size = resp_rev_buf.buf_size*2;
			received_bytes += n;
		}

		else if((n>0) && (n<resp_rev_buf.buf_left)){

			/*
			if (errno == EINTR){
				fprintf(stderr, "interrupt occurrs when read the sock \n");
				ret = -9;
				goto end4;
			}*/
			resp_rev_buf.buf_left = resp_rev_buf.buf_left - n;
			received_bytes += n;
			//break;
			continue;
		}

		else if(0 == n){
			/*为什么有一次为0的时候，什么都没有读到？？*/
			break;
		}

		else if(n < 0){

		   /* from man 2 read
		   1.It is not  an
		   error  if  this  number  is smaller than the number of bytes requested;
		   this may happen for example because fewer bytes are actually  available
		   right  now  (maybe  because we were close to end-of-file, or because we
		   are reading from a pipe, or from a terminal),  or  because  read()  was
		   interrupted  by  a  signal

		     EINTR ：
			 2.The  call  was interrupted by a signal before any data was read;
              see signal(7)

			从上面可以看出：
			 1.如果读到一部分被信号中断的话，那么会出现返回小于预期值，那么
			 这时只需要接着读就可以了
			 2.如果读取之前就被中断了的话，那么会返回错误的值EINTR，那么我们需要接着读取。
			 比如有个程序被挂起SIGUP的话，而我们的网络正在读取，那么会返回错误的值errno，如果
			 我们不判断这个值的话，我们就读取不到了。如果设置了这个值的话，那么我们在恢复运行的时候
			 返回错误，我们接着再次读取。
			).*/

			if (errno == EINTR){
				fprintf(stderr, "signal recv. read the sock\n");
				continue;
			}
			fprintf(stderr, "read the sock error\n");
			ret = -10;
			goto end4;
		}

		else{
			fprintf(stderr, "read call return`s value is unknown \n");
			ret = -11;
			goto end4;
		}
	}

	/*上面读取错误之后，malloc的缓冲区，怎么释放呢？？？？*/

	/*http_resp的char *resp_buf到底是在parse_resp_packet这个里面设置呢，还是就在这里设置呢？？？
	关于这个缓冲区，要多多考虑下，在哪里释放，parse_resp_packet这个函数的接口只要resp_p行吗？？*/

	if (
		http_set_resp_buf(
				resp_rev_buf.buf_src,
				resp_rev_buf.buf_size-resp_rev_buf.buf_left,
				resp_p
				         ) < 0
		){
		fprintf(stderr,"http_set_resp_buf error\n");
		ret = -12;
		goto end3;
	}
	//resp_p->resp_buf = resp_rev_buf.buf_src;
	/*响应报文已经接受完毕*/
	parse_resp_packet_ret = parse_resp_packet(resp_p);
	if (parse_resp_packet_ret < 0){
		fprintf(stderr,"parse_resp_packet error,error return value is %d\n",parse_resp_packet_ret);
		ret = -13;
		goto end3;
	}

	ret = 0;
	/*下面这个绝对不能少阿，不然会将这个缓冲区释放了，就变成野指针了*/
	goto end3;

	/*如果读取出错的话，在这里要释放，还是不要弄到后面取释放了，这里会泄漏内存，
	 如果读取正确的话，那么肯定要到后面取释放了*/
end4:
	if (resp_rev_buf.buf_src != NULL){
		free(resp_rev_buf.buf_src);
	}
	resp_rev_buf.buf_src = NULL;

end3:
	close(sock);
end2:
	http_uri_destroy(uri_obj_p);
end1:
	return ret;
}

void dumpNode( TidyNode tnod, int indent ,vector<string>& hrefs)
{
  TidyNode child;
  TidyAttr attr;
  string attr_value;

  for ( child = tidyGetChild(tnod); child; child = tidyGetNext(child) )
  {
    ctmbstr name;
    switch ( tidyNodeGetType(child) )
    {
    case TidyNode_Root:       name = "Root";                    break;
    case TidyNode_DocType:    name = "DOCTYPE";                 break;
    case TidyNode_Comment:    name = "Comment";                 break;
    case TidyNode_ProcIns:    name = "Processing Instruction";  break;
    case TidyNode_Text:       name = "Text";                    break;
    case TidyNode_CDATA:      name = "CDATA";                   break;
    case TidyNode_Section:    name = "XML Section";             break;
    case TidyNode_Asp:        name = "ASP";                     break;
    case TidyNode_Jste:       name = "JSTE";                    break;
    case TidyNode_Php:        name = "PHP";                     break;
    case TidyNode_XmlDecl:    name = "XML Declaration";         break;

    case TidyNode_Start:
    case TidyNode_End:
    case TidyNode_StartEnd:
    default:
      name = tidyNodeGetName( child );
      break;
    }
    assert( name != NULL );
    //printf( "indent:%d::name:%s\n", indent,name );

    attr = tidyAttrFirst(child);
    do{
		if (attr){
			if (tidyAttrName(attr) == NULL ||
				tidyAttrValue(attr) == NULL
			){
				fprintf(stderr,"tidyAttrName or tidyAttrValue return NULL\n");
			}else{

				printf("----attr name:%s::%s\n", tidyAttrName(attr),tidyAttrValue(attr));
				/*valgrind显示这里是通过没有初始化的值进行跳转*/
				if ( strncasecmp(tidyAttrName(attr), "href", strlen("href")) == 0){
					hrefs.push_back(tidyAttrValue(attr));
				}
			}
		}
    }while((attr = tidyAttrNext(attr)));

    dumpNode( child, indent + 4, hrefs);

  }
}

void dumpBody( TidyDoc tdoc , vector<string>& hrefs)
{
  dumpNode( tidyGetBody(tdoc), 0 ,hrefs);
}

void dumpDoc( TidyDoc tdoc ,vector<string>& hrefs)
{
  dumpNode( tidyGetRoot(tdoc), 0 ,hrefs);
}

int parser_html_get_hrefs(const char* html, vector<string>& hrefs)
{
	  TidyBuffer output = {0};
	  TidyBuffer errbuf = {0};
	  int rc = -1;
	  Bool ok;
	  TidyDoc tdoc = tidyCreate();                     // Initialize "document"

	  ok = tidyOptSetBool( tdoc, TidyXhtmlOut, yes );  // Convert to XHTML
	  if ( ok )
		rc = tidySetErrorBuffer( tdoc, &errbuf );      // Capture diagnostics
	  if ( rc >= 0 )
		  rc = tidyParseString( tdoc, html );           // Parse the input
	  if ( rc >= 0 )
		rc = tidyCleanAndRepair( tdoc );               // Tidy it up!
	  if ( rc >= 0 )
		rc = tidyRunDiagnostics( tdoc );               // Kvetch
	  if ( rc > 1 )                                    // If error, force output.
		rc = ( tidyOptSetBool(tdoc, TidyForceOutput, yes) ? rc : -1 );
	  if ( rc >= 0 )
		rc = tidySaveBuffer( tdoc, &output );          // Pretty Print
	   //printf("BEGIN========================\n");
	   dumpBody(tdoc, hrefs);
	   //dumpDoc(tdoc);
	  // printf("END========================\n");

	  if ( rc >= 0 ){

		 if ( rc > 0 ){
			 printf( "parser_html_get_hrefs\n");
		 }
		// printf( "\nDiagnostics:\n\n%s", errbuf.bp );
	 	// printf( "\nAnd here is the result:\n\n%s", output.bp );
	 }
	 else
		  printf( "A severe error (%d) occurred.\n", rc );

	 tidyBufFree( &output );
	 tidyBufFree( &errbuf );
	 tidyRelease( tdoc );
	 return rc;
}

void href_filter(vector<string> &hrefs)
{
	/*应该在文件类型那里也进行过滤，就是说，如果不是Html的类型，那么不进行HTML词法解析*/
	//const char *pattern = "http://([\\w-]+\\.)+[\\w-]+(/[\\w-./?%&=]*)?";
	const char *pattern = "^http://.*\\.(com|cn|org)/?$";

	/*这个是用来匹配字串，没必要看到字串的话，可以去掉*/
    //boost::cmatch mat;
	boost::regex reg_boost(pattern);
	//int pos_tmp=0;
	bool r;
	vector<string>::iterator it;
	int i=0;

	printf("hrefs.capacity:%d\n", hrefs.capacity());
	printf("hrefs.size:%d\n", hrefs.size());
	//for (it = hrefs.begin(); it != hrefs.end(); ++it, i++){
	 for (it = hrefs.begin(); it != hrefs.end();  i++){
		printf("[%d]test url\n", i);

		r = boost::regex_match((*it).c_str(), reg_boost);
		/*如果匹配失败的话*/
		if(!r){
			 //bad:hrefs:http://gongyi.qq.com/ 所以要加上/

			printf("bad:hrefs:%s\n", (*it).c_str());
			// hrefs.erase(it);
			it = hrefs.erase(it);
			continue;
		}else{

			/*在我们得到url之后，我们要判断这个url是不是已经存在系统中了*/
			if (0 == bloom_filter_query(src, (*it).c_str())){
				bloom_filter_insert(src, (*it).c_str());
				printf("======new:hrefs:%s\n", (*it).c_str());
			}else{
				printf("======old:hrefs:%s\n", (*it).c_str());
				//it = hrefs.erase(it);
				it = hrefs.erase(it);
				continue;
				/*如果已经有了，那么说明，我们不需要请求这个页面了，接着处理下一个url即可*/
			}
		}
		it++;
	}
}

/*如果buf的size不够，那么需要重新申请，并且释放buf，返回一个新的buf,并且size的大小也被改变*/
int read_handle(int sock, char** buf, int* size, int* sum)
{
	/* 应用程序传入的应该是一个指向缓冲区的指针。就是char*类型的，
	 * 这里是传指向缓冲区的指针，所以是char**,buf的值是缓冲区
	 * 指针的内存地址，*buf就可以修改缓冲区指针变量的值，**buf
	 * 就是可以修改缓冲区指向的内存单元的值，*/
	int n;
	int buf_left;
	char* re_buf=NULL;
	*sum = 0;

	buf_left = *size;

	while (1){
		errno = 0;
		n = read(sock, (void *)(*buf+*sum), buf_left);

		if (n == buf_left){
			re_buf = (char *)realloc((void *)(*buf), (*size)*2);

			if (NULL == re_buf){
				do_log(CRAWL_LOG, LOG_EMERG, LOG_FMT1, __FILE__, __LINE__,strerror(errno));
				return -1;
			}
			*buf = re_buf;
			/*剩下的刚好是*size的大小*/
			buf_left = *size;
			*size = (*size)*2;
			*sum += n;
		}

		else if((n>0) && (n<buf_left)){

			buf_left = buf_left - n;
			*sum += n;
			continue;
		}

		else if(0 == n){
			/*如果读到末尾接着读是返回什么值*/
			return 0;
		}

		else{

			if (errno ==EINTR || errno == EWOULDBLOCK || errno ==EAGAIN){
				do_log(CRAWL_LOG, LOG_ERR, LOG_FMT1, __FILE__, __LINE__,strerror(errno));
				continue;
			}
			do_log(CRAWL_LOG, LOG_ERR, LOG_FMT1, __FILE__, __LINE__,strerror(errno));
			return -2;
		}
}

int handle_ip_table_packet(const char* buf, int size, int sum)
{
	/*构建系统的二叉树*/
}

int handle_join_packet(const char* buf, int size, int sum)
{

}

int handle_exit_packet(const char* buf, int size, int sum)
{

}

int handle_packet(const char* buf, int size, int sum)
{
	/*从tracker传过来的时候计算包大小用的是unsigned int*/
	int packet_size;

	/*1字节命令加4字节包长度至少5字节*/
	if (sum < 5){
		do_log(CRAWL_LOG, LOG_ERR, LOG_FMT1, __FILE__, __LINE__,"invalid packet");
		return -1;
	}
	packet_size = 0;
	packet_size |= buf[PACKET_LENGTH_1st_POS] << 0;
	packet_size |= buf[PACKET_LENGTH_2st_POS] << 8;
	packet_size |= buf[PACKET_LENGTH_3st_POS] << 16;
	packet_size |= buf[PACKET_LENGTH_4st_POS] << 24;

	if (sum != packet_size){
		do_log(CRAWL_LOG, LOG_ERR, LOG_FMT1, __FILE__, __LINE__,"packet size is wrong");
		return -2;
	}

	switch(buf[0]){
		case RESPONSE_IP_TABLE_CMD:
			handle_ip_table_packet(buf, size,  sum);
			break;

		case BROADCAST_JOIN_CMD:
			handle_join_packet(buf, size,  sum);
			break;

		case BROADCAST_EXIT_CMD:
			handle_exit_packet(buf, size,  sum);
			break;
	}
}

/*线程的堆栈好像是2M，进程是8M？*/
void *connect2tracker_thread(void *arg)
{
	int tracker_sock = 0;
	struct sockaddr_in sin;
	int n;
	resp_packet_buf resp_rev_buf;
	int received_bytes;

	/**/
    memset(&sin, 0, sizeof(sin));
    sin.sin_family = AF_INET;
    if (inet_aton(TRACKER_IP, &sin.sin_addr) == 0){
    	do_log(CRAWL_LOG, LOG_EMERG, LOG_FMT1, __FILE__, __LINE__,"invalid IP addr");
    	exit(-1);
    }

    sin.sin_port = htons((unsigned short)TRACKER_PORT);

    tracker_sock = socket(AF_INET, SOCK_STREAM, 0);

	if (tracker_sock < 0){
		/*不能连接到tracker,那么exit直接退出进程*/
		do_log(CRAWL_LOG,LOG_EMERG,LOG_FMT1,__FILE__, __LINE__,strerror(errno));
		exit(-1);
	}
	/*这个连接和请求HTTP不同，它不需要select定时*/
	if (connect(tracker_sock, (struct sockaddr *)&sin, sizeof(sin)) < 0){
		do_log(CRAWL_LOG,LOG_EMERG,LOG_FMT1,__FILE__, __LINE__,strerror(errno));
		exit(-1);
	}

	resp_rev_buf.buf_size = DEFAULT_IP_TABLE_RESP_REV_BUF_SIZE;
	resp_rev_buf.buf_src  = NULL;
	resp_rev_buf.buf_left = DEFAULT_IP_TABLE_RESP_REV_BUF_SIZE;

	resp_rev_buf.buf_src  = (char *)malloc(DEFAULT_IP_TABLE_RESP_REV_BUF_SIZE);
	if (NULL == resp_rev_buf.buf_src){
		do_log(CRAWL_LOG,LOG_EMERG,LOG_FMT1,__FILE__, __LINE__,strerror(errno));
		exit(-1);
	}

	/*解析tracker报文*/
	while(1){
		received_bytes = 0;
		n = read_handle(tracker_sock, &(resp_rev_buf.buf_src), &(resp_rev_buf.buf_size), &received_bytes);
		if (-1 == n){
			/*这里应该结束还是*/
			exit(-1);
		}
		handle_packet();



	}


}
void *crawl_thread(void *arg)
{
	struct timespec tm;
	struct timeval now;
	struct timezone tz;
	string url;
	string url_tmp;
	string page_buf;
	vector<string> hrefs;
	vector<string>::iterator it;
	struct html_parse *hp = NULL;

//多定义一个int变量就出错了
#if 0
    boost::cmatch mat;
	//boost::regex reg_boost(pattern);
    int a;
    int b;
    int c;
	int pos_tmp=0;
	bool r;
#endif

//	regmatch_t pm[25];
//	const size_t nmatch = sizeof(pm) / sizeof(pm[0]);
//	int reg_ret = 0;
//	regex_t reg;
//	char ebuf[128];
//	pcre *re;
//	const char *error;
//	int erroffset;
//	int ovector[OVECCOUNT];
//	int rc;
//	int i;

	http_resp resp;
	char *not_const_url = NULL;
	int ret = 0;
	int brk_flag=0;

	while(1){

		brk_flag = 0;
		http_resp_init(&resp);

		pthread_mutex_lock(&url_mutex);
		while(url_buff.size() == 0){

			//typedef unsigned long int pthread_t;
			fprintf(stderr,"pthread_t:%lu url_buff.size is 0,waiting\n",pthread_self());
			gettimeofday(&now,&tz);
			tm.tv_sec = now.tv_sec + 5;
			pthread_cond_timedwait(&url_not_empty, &url_mutex, &tm);
		}
		//url = url_buff.back();
		url = url_buff.front();
		//这里应该弹出头部，新增数据应该插入到尾部去
		url_buff.pop();
		pthread_mutex_unlock(&url_mutex);

		/*接着通过这个url来获取更多的url，就是说需要请求这个url页面的内容*/

		/*****************下面这样转换行吗？？？*************************/
		/*其实，下面这样直接转换从const到非const应该是非常危险的，本来别人不能修改，，而你偏要转换成可以，
		会出现什么问题呢？？当然，这个函数后面并没有直接修改传入的参数,但是如果修改了呢？那会出现未知的错误*/

		not_const_url = strndup(url.c_str(), url.length());

		if (NULL == not_const_url){
			fprintf(stderr,"strndup url error\n");
			ret = 1;
			/*内存都分配不了了，没有必要继续下去了，直接退出了*/
			brk_flag = 1;
			goto end1;
		}

		if (get_page_through_url(not_const_url, &resp) < 0 ){

			/*这里的返回值可能会有malloc的错误，线程还是不退出么？？*/
			fprintf(stderr ,"get page error\n");
			goto end1;
			//continue;
		}

		/*当前的情况是，一运行就会错，得不到期望的字节数，但是单步的话，不会错，但是比wget获得的还多？？*/
		/*当取得了页面缓存之后，下面进行正则表达式的匹配，找出更多的URL*/

		hp = html_newparser();

		if (NULL == hp){
			fprintf(stderr, "alloc parser error!\n");
			//continue;
			ret = 2;
			/*内存都分配不了了，没有必要继续下去了，直接退出了*/
			brk_flag = 1;
			goto end1;
		}

		parser_html_get_hrefs(resp.content.body, hrefs);
		//html_parser(hp, resp.content.body, resp.content.body_length);

		/*从HTML中过滤超链接，那我们在哪里过滤图片等东西呢？？*/
		href_filter(hrefs);

		printf("href filter finished\n");
		pthread_mutex_lock(&url_mutex);

		for (it = hrefs.begin(); it != hrefs.end(); ++it){

			url_buff.push(*it);
			*it = *it + '\n';
			/*接着要将超链接写入文件中，并且设置bloomfilter,这里应该要和压入数据同时锁定吧？
			  重复的应该已经被href_filter过滤掉了*/
			write(fileno(fp2), (*it).c_str(), (*it).length());
		}
		pthread_mutex_unlock(&url_mutex);
		pthread_cond_signal(&url_not_empty);

		html_freeparser(hp);
end1:
		if(not_const_url != NULL){
			free(not_const_url);
		}
		not_const_url = NULL;

		/*释放resp中的缓冲区*/
		/*下面有个释放的一片缓冲区的函数，这片缓冲区从哪里来阿？？？？？？？？？？？？？？？？？？？？？？？？？？？？？*/
		if (http_resp_destroy(&resp) < 0){
			fprintf(stderr ,"http_resp_destroy error\n");
			continue;
		}

		if (1 == brk_flag){
			break;
		}
	}
	return (void *)ret;
}

struct hostent *gethostnameinfo(http_uri *uri_obj_p, const char *host)
{
    struct hostent hostbuf,*hp;
    size_t hstbuflen;
    char tmphstbuf[8192];
    int res;
    int herr;

    res = gethostbyname_r(host, &hostbuf, tmphstbuf, sizeof(tmphstbuf), &hp,&herr);

    if (res || NULL == hp){
    	fprintf(stderr, "gethostnameinfo error\n");
		return NULL;
    }else{
    	fprintf(stderr, "gethostnameinfo success\n");
		return hp;
    }
}

int main(int argc, char *argv[])
{
    pthread_t crawl_tid[MAX_PTHREAD_NUM];
    pthread_t tracker_tid;
    int err;
	struct stat statbuf;
	int fd1;

	err = pthread_create(&tracker_tid, NULL, connect2tracker_thread, NULL);
	if (err != 0){

	}

    for (int i=0; i<MAX_PTHREAD_NUM; i++){
		err = pthread_create(&crawl_tid[i], NULL, crawl_thread, NULL);
		if (err != 0){
			fprintf(stderr, "create pthread[%d] error\n", i);
			break;
		}
   	 }

    if (argc != 2){
        fprintf(stderr,"the program needs a URL to begin searching\n");
		exit(EXIT_FAILURE);
    }

	fd1 = open("bloomfilter", O_RDWR);
	if (0 == fd1){
		perror("open file:bloomfilter error\n");
		exit(EXIT_FAILURE);
	}

	fp2 = fopen("url","a");
	if (NULL == fp2){
		perror("open file:url error\n");
		goto err_handle2;
	}

	if (fstat(fd1, &statbuf) < 0){
		perror("fstat bloomfilter error\n");
		goto err_handle1;
	}

	/*由字节大小转化为位大小,0byte-0bit|1,2...8bit-1byte|9,10...bit-2byte*/
	bloom_filter_set_hash_table_size((statbuf.st_size + 7) / 8);
	//hash_table_size = (statbuf.st_size + 7) / 8;

	if( (src = mmap(0, statbuf.st_size, PROT_READ|PROT_WRITE, MAP_SHARED,fd1, 0)) == MAP_FAILED ){
		fprintf(stderr ,"mmap error\n");
		goto err_handle1;
	}

	pthread_mutex_lock(&url_mutex);

	if ( bloom_filter_query(src, argv[1]) < 0){
		fprintf(stderr ,"write first url error\n");
		goto err_handle1;
	}
	url_buff.push(argv[1]);

	if (fputs(argv[1], fp2) < 0){
		fprintf(stderr ,"write first url error\n");
		goto err_handle1;
	}
	fputs("\n", fp2);
	//如果，这里不关闭的话，程序执行到这里来，是看不到url文件中有什么记录的
	//fclose(fp2);
	pthread_mutex_unlock(&url_mutex);

	pthread_cond_signal(&url_not_empty);

	for (int k=0; k<MAX_PTHREAD_NUM; k++){
		err = pthread_join(crawl_tid[k], NULL);
		if (err != 0){
			fprintf(stderr, "join pthread[%d] error\n",k);
		}
	}
	printf("program END\n");
	close(fd1);
	fclose(fp2);
	return 0;

err_handle1:
	fclose(fp2);
err_handle2:
	close(fd1);
	exit(EXIT_FAILURE);
	/*
	//这里应该写道磁盘上去的
    //根据传入的URL获取链接，然后再用这些链接获得网页再打开新的链接
    struct sockaddr_in sin;
    http_uri *uri_obj_p = NULL;

    memset(&sin, 0, sizeof(sin));
    sin.sin_family = AF_INET;
    sin.sin_port   = htons((unsigned short)80);

    uri_obj_p = http_uri_new();
    if (NULL == uri_obj_p){
    	fprintf(stderr,"http_uri_new error\n");
		close(fd1);
		close(fp2);
		exit(EXIT_FAILURE);
    }

    if (-1 == http_uri_parse(argv[1], uri_obj_p) ){
    	http_uri_destroy(uri_obj_p);
		fprintf(stderr, "parse the URL error\n");
		close(fd1);
		close(fp2);
		exit(EXIT_FAILURE);
    }
	*/
}



/*
while( (n = read(sock, rev_buf, sizeof(rev_buf)-1)) >0 ){

	rev_buf[sizeof(rev_buf)-1] = '\0';

	如果是第一帧的话，那么我们需要得到下面的数据,!!!一定是在第一帧里面吗？？？
	if (isfirst_frame){

		memset(header_content_buf, 0, sizeof(header_content_buf));
		128字节的最长首部，减去一个\r,一个\n,宽度为126，这里一定要指定%126.不然会有溢出，
		会收到攻击，其实这里也可以不用硬编码成126,sscanf成功的话，返回参数个数，就是1

		if (sscanf(rev_buf, "%126s\r\n", header_content_buf) <= 0){
			fprintf(stderr, "sscanf rev_buf error\n");
			close(sock);
			http_uri_destroy(uri_obj_p);
			return -7;
		}

		这里应该只搜索第一行的

		if ( (NULL == strstr(header_content_buf,"200 OK"))
		  && (NULL == strstr(header_content_buf,"200 Ok"))
		  && (NULL == strstr(header_content_buf,"200 ok"))
		   ){
		    fprintf(stderr, "search 200 OK error\n");
			close(sock);
			http_uri_destroy(uri_obj_p);
			return -8;
		}
		if ( (NULL == strstr(rev_buf,"200 OK"))
		  && (NULL == strstr(rev_buf,"200 Ok"))
		  && (NULL == strstr(rev_buf,"200 ok"))
		  ){
			fprintf(stderr, "search 200 OK error\n");
			close(sock);
			http_uri_destroy(uri_obj_p);
			return -8;
		}

		content_length_start = NULL;
		content_length_end   = NULL;
		忽略header_content_buf\r\n首部行，从它后面开始搜索
		//content_length_start = strstr(rev_buf + strlen(header_content_buf) + 2 ,"Content-Length: ");
		//长度字段不一定会有，那么怎么获得长度呢？
		content_length_start = strstr(rev_buf,"Content-Length: ");
		if (!content_length_start){
			fprintf(stderr, "find Content_length error\n");
			close(sock);
			http_uri_destroy(uri_obj_p);
			return -9;
		}
		content_length_start += strlen("Content-Length: ");//sizeof要比strlen大一个

		content_length_end   = strstr(content_length_start, "\r\n");
		if (!content_length_end){
			fprintf(stderr, "find Content-length start error\n");
			close(sock);
			http_uri_destroy(uri_obj_p);
			return -10;
		}
		memset(content_length_buf,0,sizeof(content_length_buf));

		if ( (content_length_end - content_length_start) > (sizeof(content_length_buf)-1) ){

			fprintf(stderr, "content length string is too long \n");
			close(sock);
			http_uri_destroy(uri_obj_p);
			return -11;
		}
		memcpy(content_length_buf, content_length_start, content_length_end - content_length_start);
		content_length = atoi(content_length_buf);
		isfirst_frame = false;
	}

	if (!data_found){
		data_start = NULL;
		这里能用content_length_end吗？不行，因为数据可能会在第二帧里面
		data_start = strstr(rev_buf, "\r\n\r\n");

		if (data_start){
			data_found = true;
			data_start += strlen("\r\n\r\n");
			write_no = n - (data_start - rev_buf);
			rev_buf[n] = '\0';
			page_buf += data_start;

		}else{
			//如果这里没有找到\r\n\r\n的话，那么我们可以直接操作下一个链接了而不是接着read
			fprintf(stderr, "can not find data start \n");
			close(sock);
			http_uri_destroy(uri_obj_p);
			return -13;
		}
	}else{
		rev_buf[n] = '\0';
		page_buf += rev_buf;
	}
}*/

#if 0
		pos_tmp=0;
		//r = boost::regex_match(resp.content.body + position,mat,reg_boost);
		while(boost::regex_search(resp.content.body + pos_tmp,mat,reg_boost)){
			pos_tmp = mat[0].length();
			cout<<mat[0]<<endl;
		}
#endif
	#if 0
		re = pcre_compile(pattern,       // pattern, 输入参数，将要被编译的字符串形式的正则表达式
						   0,            // options, 输入参数，用来指定编译时的一些选项
						   &error,       // errptr, 输出参数，用来输出错误信息
						   &erroffset,   // erroffset, 输出参数，pattern中出错位置的偏移量
						   NULL);        // tableptr, 输入参数，用来指定字符表，一般情况用NULL
		 // 返回值：被编译好的正则表达式的pcre内部表示结构
		 if (re == NULL) {                 //如果编译失败，返回错误信息
			 fprintf(stderr,"PCRE compilation failed at offset %d: %s/n", erroffset, error);
			 continue;
		 }
		 rc = pcre_exec(re,            // code, 输入参数，用pcre_compile编译好的正则表达结构的指针
						NULL,          // extra, 输入参数，用来向pcre_exec传一些额外的数据信息的结构的指针
						resp.content.body,           // subject, 输入参数，要被用来匹配的字符串
						resp.content.body_length,   // length, 输入参数， 要被用来匹配的字符串的指针
						0,             // startoffset, 输入参数，用来指定subject从什么位置开始被匹配的偏移量
						0,             // options, 输入参数， 用来指定匹配过程中的一些选项
						ovector,       // ovector, 输出参数，用来返回匹配位置偏移量的数组
						OVECCOUNT);    // ovecsize, 输入参数， 用来返回匹配位置偏移量的数组的最大大小
		// 返回值：匹配成功返回非负数，没有匹配返回负数
		 if (rc < 0) {                     //如果没有匹配，返回错误信息
			 if (rc == PCRE_ERROR_NOMATCH){
				 printf("Sorry, no match .../n");
			 }
			 else{
				 printf("Matching error %d/n", rc);
			 }
			 pcre_free(re);
			 continue;
		 }
		 printf("/nOK, has matched .../n/n");   //没有出错，已经匹配

		 for (i = 0; i < rc; i++) {             //分别取出捕获分组 $0整个正则公式 $1第一个()
			 char *substring_start = resp.content.body + ovector[2*i];
			 int substring_length = ovector[2*i+1] - ovector[2*i];
			 printf("%2d: %d,%50s/n", i, substring_length, substring_start);
		 }
		 pcre_free(re);                     // 编译正则表达式re 释放内存
#endif

#if 0
		 /*构建正则表达式*/
		reg_ret = regcomp(&reg ,pattern, REG_EXTENDED);
		if (0 != reg_ret){
			regerror(reg_ret, &reg, ebuf, sizeof(ebuf));
			fprintf(stderr, "%s, pattern '%s'\n",ebuf ,pattern);
			/*到底是继续还是exit，这里exit直接进程全部退出了！！！*/
			continue;
		}

		reg_ret = regexec(&reg, resp.content.body, nmatch, pm, 0);
		if (reg_ret == REG_NOMATCH){
			fprintf(stderr, "cat not match the pattern\n");
			continue;

		}else{
			//regerror(reg_ret, &reg, ebuf, sizeof(ebuf));
			//fprintf(stderr, "regexec error\n");
			//;
		}

		for (i=0; (i<nmatch) && (pm[i].rm_so != -1); i++){

			pthread_mutex_lock(&url_mutex);
			url_tmp.clear();
		//	url_tmp = '\n' + page_buf.substr(pm[i].rm_so, pm[i].rm_eo-pm[i].rm_so + 1);
			url_tmp = '\n' + page_buf.substr(pm[i].rm_so, pm[i].rm_eo);

			pthread_mutex_unlock(&url_mutex);

			/*fputs会加上换行符吗？*/
			fputs(url_tmp.c_str(),fp2);
		}
#endif

#if 0
		/*在我们得到url之后，我们要判断这个url是不是已经存在系统中了*/
		if (0 == bloom_filter_query(src, url.c_str())){

			/*多个线程同时写URL到文件中会不会出现竞争，好像linux里面添加APPEND模式进行写的话，不会出现*/
			if (fputs(url.c_str(), fp2) < 0){
				fprintf(stderr ,"write url error\n");
				continue;
				//	close(fp2);
				//	exit(EXIT_FAILURE);
			}
			bloom_filter_insert(src, url.c_str());
		}else{
			/*如果已经有了，那么说明，我们不需要请求这个页面了，接着处理下一个url即可*/
			continue;
		}
#endif
